{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ac56847586905b36",
   "metadata": {},
   "source": [
    "MRC mechine reading comprehension 机器阅读理解\n",
    "1. `return_offsets_mapping=True` 返回分词偏移映射\n",
    "2. `sequence_ids`\n",
    "3."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cfdaa144adff8331",
   "metadata": {},
   "source": [
    "导入包"
   ]
  },
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2025-05-20T13:51:52.059212Z",
     "start_time": "2025-05-20T13:51:47.615934Z"
    }
   },
   "source": [
    "from token import AWAIT\n",
    "\n",
    "from datasets import load_dataset\n",
    "from transformers import AutoTokenizer, AutoModelForQuestionAnswering, Trainer, TrainingArguments, DefaultDataCollator\n",
    "from MyHelper import *\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\51165\\.conda\\envs\\e12\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "execution_count": 1
  },
  {
   "cell_type": "markdown",
   "id": "d11c54e1396b90b1",
   "metadata": {},
   "source": [
    "加载数据源"
   ]
  },
  {
   "cell_type": "code",
   "id": "4af93d31a9394354",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:07.104818Z",
     "start_time": "2025-05-20T13:51:52.160521Z"
    }
   },
   "source": [
    "datasets = load_dataset(\"cmrc2018\", trust_remote_code=True)\n",
    "datasets"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['id', 'context', 'question', 'answers'],\n",
       "        num_rows: 10142\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['id', 'context', 'question', 'answers'],\n",
       "        num_rows: 3219\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['id', 'context', 'question', 'answers'],\n",
       "        num_rows: 1002\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "id": "46006581164bf7c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:07.182449Z",
     "start_time": "2025-05-20T13:52:07.179365Z"
    }
   },
   "source": "print(datasets[\"train\"][0])",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'id': 'TRAIN_186_QUERY_0', 'context': '范廷颂枢机（，），圣名保禄·若瑟（），是越南罗马天主教枢机。1963年被任为主教；1990年被擢升为天主教河内总教区宗座署理；1994年被擢升为总主教，同年年底被擢升为枢机；2009年2月离世。范廷颂于1919年6月15日在越南宁平省天主教发艳教区出生；童年时接受良好教育后，被一位越南神父带到河内继续其学业。范廷颂于1940年在河内大修道院完成神学学业。范廷颂于1949年6月6日在河内的主教座堂晋铎；及后被派到圣女小德兰孤儿院服务。1950年代，范廷颂在河内堂区创建移民接待中心以收容到河内避战的难民。1954年，法越战争结束，越南民主共和国建都河内，当时很多天主教神职人员逃至越南的南方，但范廷颂仍然留在河内。翌年管理圣若望小修院；惟在1960年因捍卫修院的自由、自治及拒绝政府在修院设政治课的要求而被捕。1963年4月5日，教宗任命范廷颂为天主教北宁教区主教，同年8月15日就任；其牧铭为「我信天主的爱」。由于范廷颂被越南政府软禁差不多30年，因此他无法到所属堂区进行牧灵工作而专注研读等工作。范廷颂除了面对战争、贫困、被当局迫害天主教会等问题外，也秘密恢复修院、创建女修会团体等。1990年，教宗若望保禄二世在同年6月18日擢升范廷颂为天主教河内总教区宗座署理以填补该教区总主教的空缺。1994年3月23日，范廷颂被教宗若望保禄二世擢升为天主教河内总教区总主教并兼天主教谅山教区宗座署理；同年11月26日，若望保禄二世擢升范廷颂为枢机。范廷颂在1995年至2001年期间出任天主教越南主教团主席。2003年4月26日，教宗若望保禄二世任命天主教谅山教区兼天主教高平教区吴光杰主教为天主教河内总教区署理主教；及至2005年2月19日，范廷颂因获批辞去总主教职务而荣休；吴光杰同日真除天主教河内总教区总主教职务。范廷颂于2009年2月22日清晨在河内离世，享年89岁；其葬礼于同月26日上午在天主教河内总教区总主教座堂举行。', 'question': '范廷颂是什么时候被任为主教的？', 'answers': {'text': ['1963年'], 'answer_start': [30]}}\n"
     ]
    }
   ],
   "execution_count": 3
  },
  {
   "cell_type": "markdown",
   "id": "9c20ebf1",
   "metadata": {},
   "source": [
    "数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "id": "9a496ef89d623d22",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:07.600163Z",
     "start_time": "2025-05-20T13:52:07.272121Z"
    }
   },
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(Config.hfl_chinese_macbert_base)\n",
    "tokenizer"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BertTokenizerFast(name_or_path='hfl/chinese-macbert-base', vocab_size=21128, model_max_length=1000000000000000019884624838656, is_fast=True, padding_side='right', truncation_side='right', special_tokens={'unk_token': '[UNK]', 'sep_token': '[SEP]', 'pad_token': '[PAD]', 'cls_token': '[CLS]', 'mask_token': '[MASK]'}, clean_up_tokenization_spaces=False, added_tokens_decoder={\n",
       "\t0: AddedToken(\"[PAD]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t100: AddedToken(\"[UNK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t101: AddedToken(\"[CLS]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t102: AddedToken(\"[SEP]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "\t103: AddedToken(\"[MASK]\", rstrip=False, lstrip=False, single_word=False, normalized=False, special=True),\n",
       "}\n",
       ")"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "id": "d5bd9187",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:20.040258Z",
     "start_time": "2025-05-20T13:52:20.031771Z"
    }
   },
   "source": [
    "sample_dataset = datasets[\"train\"].select(range(2))\n",
    "tokenized_examples = tokenizer(text=sample_dataset[\"question\"], text_pair=sample_dataset[\"context\"], max_length=512, truncation=\"only_second\", return_offsets_mapping=True)\n",
    "tokenized_examples.keys()"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['input_ids', 'token_type_ids', 'attention_mask', 'offset_mapping'])"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:23.160181Z",
     "start_time": "2025-05-20T13:52:23.156822Z"
    }
   },
   "cell_type": "code",
   "source": [
    "for k in tokenized_examples.keys():\n",
    "    print(k, \":\", tokenized_examples[k][0])"
   ],
   "id": "3c9f7c45788e4402",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_ids : [101, 5745, 2455, 7563, 3221, 784, 720, 3198, 952, 6158, 818, 711, 712, 3136, 4638, 8043, 102, 5745, 2455, 7563, 3364, 3322, 8020, 8024, 8021, 8024, 1760, 1399, 924, 4882, 185, 5735, 4449, 8020, 8021, 8024, 3221, 6632, 1298, 5384, 7716, 1921, 712, 3136, 3364, 3322, 511, 9155, 2399, 6158, 818, 711, 712, 3136, 8039, 8431, 2399, 6158, 3091, 1285, 711, 1921, 712, 3136, 3777, 1079, 2600, 3136, 1277, 2134, 2429, 5392, 4415, 8039, 8447, 2399, 6158, 3091, 1285, 711, 2600, 712, 3136, 8024, 1398, 2399, 2399, 2419, 6158, 3091, 1285, 711, 3364, 3322, 8039, 8170, 2399, 123, 3299, 4895, 686, 511, 5745, 2455, 7563, 754, 9915, 2399, 127, 3299, 8115, 3189, 1762, 6632, 1298, 2123, 2398, 4689, 1921, 712, 3136, 1355, 5683, 3136, 1277, 1139, 4495, 8039, 4997, 2399, 3198, 2970, 1358, 5679, 1962, 3136, 5509, 1400, 8024, 6158, 671, 855, 6632, 1298, 4868, 4266, 2372, 1168, 3777, 1079, 5326, 5330, 1071, 2110, 689, 511, 5745, 2455, 7563, 754, 9211, 2399, 1762, 3777, 1079, 1920, 934, 6887, 7368, 2130, 2768, 4868, 2110, 2110, 689, 511, 5745, 2455, 7563, 754, 8594, 2399, 127, 3299, 127, 3189, 1762, 3777, 1079, 4638, 712, 3136, 2429, 1828, 3232, 7195, 8039, 1350, 1400, 6158, 3836, 1168, 1760, 1957, 2207, 2548, 1065, 2109, 1036, 7368, 3302, 1218, 511, 8707, 2399, 807, 8024, 5745, 2455, 7563, 1762, 3777, 1079, 1828, 1277, 1158, 2456, 4919, 3696, 2970, 2521, 704, 2552, 809, 3119, 2159, 1168, 3777, 1079, 6912, 2773, 4638, 7410, 3696, 511, 9258, 2399, 8024, 3791, 6632, 2773, 751, 5310, 3338, 8024, 6632, 1298, 3696, 712, 1066, 1469, 1744, 2456, 6963, 3777, 1079, 8024, 2496, 3198, 2523, 1914, 1921, 712, 3136, 4868, 5466, 782, 1447, 6845, 5635, 6632, 1298, 4638, 1298, 3175, 8024, 852, 5745, 2455, 7563, 793, 4197, 4522, 1762, 3777, 1079, 511, 5422, 2399, 5052, 4415, 1760, 5735, 3307, 2207, 934, 7368, 8039, 2668, 1762, 8779, 2399, 1728, 2932, 1310, 934, 7368, 4638, 5632, 4507, 510, 5632, 3780, 1350, 2867, 5318, 3124, 2424, 1762, 934, 7368, 6392, 3124, 3780, 6440, 4638, 6206, 3724, 5445, 6158, 2936, 511, 9155, 2399, 125, 3299, 126, 3189, 8024, 3136, 2134, 818, 1462, 5745, 2455, 7563, 711, 1921, 712, 3136, 1266, 2123, 3136, 1277, 712, 3136, 8024, 1398, 2399, 129, 3299, 8115, 3189, 2218, 818, 8039, 1071, 4288, 7208, 711, 519, 2769, 928, 1921, 712, 4638, 4263, 520, 511, 4507, 754, 5745, 2455, 7563, 6158, 6632, 1298, 3124, 2424, 6763, 4881, 2345, 679, 1914, 8114, 2399, 8024, 1728, 3634, 800, 3187, 3791, 1168, 2792, 2247, 1828, 1277, 6822, 6121, 4288, 4130, 2339, 868, 5445, 683, 3800, 4777, 6438, 5023, 2339, 868, 511, 5745, 2455, 7563, 7370, 749, 7481, 2190, 2773, 751, 510, 6577, 1737, 510, 6158, 2496, 2229, 6833, 2154, 1921, 712, 3136, 833, 5023, 7309, 7579, 1912, 8024, 738, 4908, 2166, 2612, 1908, 934, 7368, 510, 1158, 2456, 1957, 934, 833, 1730, 860, 5023, 511, 8431, 2399, 8024, 3136, 2134, 5735, 3307, 924, 4882, 753, 686, 1762, 1398, 2399, 127, 3299, 8123, 3189, 3091, 1285, 5745, 2455, 7563, 711, 1921, 712, 3136, 3777, 1079, 2600, 3136, 1277, 2134, 2429, 5392, 102]\n",
      "token_type_ids : [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
      "attention_mask : [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
      "offset_mapping : [(0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (15, 16), (16, 17), (17, 18), (18, 19), (19, 20), (20, 21), (21, 22), (22, 23), (23, 24), (24, 25), (25, 26), (26, 27), (27, 28), (28, 29), (29, 30), (30, 34), (34, 35), (35, 36), (36, 37), (37, 38), (38, 39), (39, 40), (40, 41), (41, 45), (45, 46), (46, 47), (47, 48), (48, 49), (49, 50), (50, 51), (51, 52), (52, 53), (53, 54), (54, 55), (55, 56), (56, 57), (57, 58), (58, 59), (59, 60), (60, 61), (61, 62), (62, 63), (63, 67), (67, 68), (68, 69), (69, 70), (70, 71), (71, 72), (72, 73), (73, 74), (74, 75), (75, 76), (76, 77), (77, 78), (78, 79), (79, 80), (80, 81), (81, 82), (82, 83), (83, 84), (84, 85), (85, 86), (86, 87), (87, 91), (91, 92), (92, 93), (93, 94), (94, 95), (95, 96), (96, 97), (97, 98), (98, 99), (99, 100), (100, 101), (101, 105), (105, 106), (106, 107), (107, 108), (108, 110), (110, 111), (111, 112), (112, 113), (113, 114), (114, 115), (115, 116), (116, 117), (117, 118), (118, 119), (119, 120), (120, 121), (121, 122), (122, 123), (123, 124), (124, 125), (125, 126), (126, 127), (127, 128), (128, 129), (129, 130), (130, 131), (131, 132), (132, 133), (133, 134), (134, 135), (135, 136), (136, 137), (137, 138), (138, 139), (139, 140), (140, 141), (141, 142), (142, 143), (143, 144), (144, 145), (145, 146), (146, 147), (147, 148), (148, 149), (149, 150), (150, 151), (151, 152), (152, 153), (153, 154), (154, 155), (155, 156), (156, 157), (157, 158), (158, 159), (159, 163), (163, 164), (164, 165), (165, 166), (166, 167), (167, 168), (168, 169), (169, 170), (170, 171), (171, 172), (172, 173), (173, 174), (174, 175), (175, 176), (176, 177), (177, 178), (178, 179), (179, 180), (180, 181), (181, 182), (182, 186), (186, 187), (187, 188), (188, 189), (189, 190), (190, 191), (191, 192), (192, 193), (193, 194), (194, 195), (195, 196), (196, 197), (197, 198), (198, 199), (199, 200), (200, 201), (201, 202), (202, 203), (203, 204), (204, 205), (205, 206), (206, 207), (207, 208), (208, 209), (209, 210), (210, 211), (211, 212), (212, 213), (213, 214), (214, 215), (215, 216), (216, 217), (217, 218), (218, 222), (222, 223), (223, 224), (224, 225), (225, 226), (226, 227), (227, 228), (228, 229), (229, 230), (230, 231), (231, 232), (232, 233), (233, 234), (234, 235), (235, 236), (236, 237), (237, 238), (238, 239), (239, 240), (240, 241), (241, 242), (242, 243), (243, 244), (244, 245), (245, 246), (246, 247), (247, 248), (248, 249), (249, 250), (250, 251), (251, 252), (252, 253), (253, 257), (257, 258), (258, 259), (259, 260), (260, 261), (261, 262), (262, 263), (263, 264), (264, 265), (265, 266), (266, 267), (267, 268), (268, 269), (269, 270), (270, 271), (271, 272), (272, 273), (273, 274), (274, 275), (275, 276), (276, 277), (277, 278), (278, 279), (279, 280), (280, 281), (281, 282), (282, 283), (283, 284), (284, 285), (285, 286), (286, 287), (287, 288), (288, 289), (289, 290), (290, 291), (291, 292), (292, 293), (293, 294), (294, 295), (295, 296), (296, 297), (297, 298), (298, 299), (299, 300), (300, 301), (301, 302), (302, 303), (303, 304), (304, 305), (305, 306), (306, 307), (307, 308), (308, 309), (309, 310), (310, 311), (311, 312), (312, 313), (313, 314), (314, 315), (315, 316), (316, 317), (317, 318), (318, 319), (319, 320), (320, 321), (321, 325), (325, 326), (326, 327), (327, 328), (328, 329), (329, 330), (330, 331), (331, 332), (332, 333), (333, 334), (334, 335), (335, 336), (336, 337), (337, 338), (338, 339), (339, 340), (340, 341), (341, 342), (342, 343), (343, 344), (344, 345), (345, 346), (346, 347), (347, 348), (348, 349), (349, 350), (350, 351), (351, 352), (352, 353), (353, 354), (354, 355), (355, 356), (356, 360), (360, 361), (361, 362), (362, 363), (363, 364), (364, 365), (365, 366), (366, 367), (367, 368), (368, 369), (369, 370), (370, 371), (371, 372), (372, 373), (373, 374), (374, 375), (375, 376), (376, 377), (377, 378), (378, 379), (379, 380), (380, 381), (381, 382), (382, 383), (383, 384), (384, 385), (385, 386), (386, 387), (387, 388), (388, 390), (390, 391), (391, 392), (392, 393), (393, 394), (394, 395), (395, 396), (396, 397), (397, 398), (398, 399), (399, 400), (400, 401), (401, 402), (402, 403), (403, 404), (404, 405), (405, 406), (406, 407), (407, 408), (408, 409), (409, 410), (410, 411), (411, 412), (412, 413), (413, 414), (414, 415), (415, 416), (416, 417), (417, 418), (418, 419), (419, 420), (420, 421), (421, 422), (422, 424), (424, 425), (425, 426), (426, 427), (427, 428), (428, 429), (429, 430), (430, 431), (431, 432), (432, 433), (433, 434), (434, 435), (435, 436), (436, 437), (437, 438), (438, 439), (439, 440), (440, 441), (441, 442), (442, 443), (443, 444), (444, 445), (445, 446), (446, 447), (447, 448), (448, 449), (449, 450), (450, 451), (451, 452), (452, 453), (453, 454), (454, 455), (455, 456), (456, 457), (457, 458), (458, 459), (459, 460), (460, 461), (461, 462), (462, 463), (463, 464), (464, 465), (465, 466), (466, 467), (467, 468), (468, 469), (469, 470), (470, 471), (471, 472), (472, 473), (473, 474), (474, 475), (475, 476), (476, 477), (477, 478), (478, 479), (479, 480), (480, 481), (481, 482), (482, 483), (483, 484), (484, 485), (485, 486), (486, 487), (487, 488), (488, 489), (489, 490), (490, 491), (491, 492), (492, 493), (493, 494), (494, 495), (495, 499), (499, 500), (500, 501), (501, 502), (502, 503), (503, 504), (504, 505), (505, 506), (506, 507), (507, 508), (508, 509), (509, 510), (510, 511), (511, 512), (512, 513), (513, 514), (514, 516), (516, 517), (517, 518), (518, 519), (519, 520), (520, 521), (521, 522), (522, 523), (523, 524), (524, 525), (525, 526), (526, 527), (527, 528), (528, 529), (529, 530), (530, 531), (531, 532), (532, 533), (533, 534), (0, 0)]\n"
     ]
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:23.203438Z",
     "start_time": "2025-05-20T13:52:23.198293Z"
    }
   },
   "cell_type": "code",
   "source": "\"|\".join([tokenizer.decode(id) for id in tokenized_examples[\"input_ids\"][0]])",
   "id": "56f5cb951e1b2c9e",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'[CLS]|范|廷|颂|是|什|么|时|候|被|任|为|主|教|的|？|[SEP]|范|廷|颂|枢|机|（|，|）|，|圣|名|保|禄|·|若|瑟|（|）|，|是|越|南|罗|马|天|主|教|枢|机|。|1963|年|被|任|为|主|教|；|1990|年|被|擢|升|为|天|主|教|河|内|总|教|区|宗|座|署|理|；|1994|年|被|擢|升|为|总|主|教|，|同|年|年|底|被|擢|升|为|枢|机|；|2009|年|2|月|离|世|。|范|廷|颂|于|1919|年|6|月|15|日|在|越|南|宁|平|省|天|主|教|发|艳|教|区|出|生|；|童|年|时|接|受|良|好|教|育|后|，|被|一|位|越|南|神|父|带|到|河|内|继|续|其|学|业|。|范|廷|颂|于|1940|年|在|河|内|大|修|道|院|完|成|神|学|学|业|。|范|廷|颂|于|1949|年|6|月|6|日|在|河|内|的|主|教|座|堂|晋|铎|；|及|后|被|派|到|圣|女|小|德|兰|孤|儿|院|服|务|。|1950|年|代|，|范|廷|颂|在|河|内|堂|区|创|建|移|民|接|待|中|心|以|收|容|到|河|内|避|战|的|难|民|。|1954|年|，|法|越|战|争|结|束|，|越|南|民|主|共|和|国|建|都|河|内|，|当|时|很|多|天|主|教|神|职|人|员|逃|至|越|南|的|南|方|，|但|范|廷|颂|仍|然|留|在|河|内|。|翌|年|管|理|圣|若|望|小|修|院|；|惟|在|1960|年|因|捍|卫|修|院|的|自|由|、|自|治|及|拒|绝|政|府|在|修|院|设|政|治|课|的|要|求|而|被|捕|。|1963|年|4|月|5|日|，|教|宗|任|命|范|廷|颂|为|天|主|教|北|宁|教|区|主|教|，|同|年|8|月|15|日|就|任|；|其|牧|铭|为|「|我|信|天|主|的|爱|」|。|由|于|范|廷|颂|被|越|南|政|府|软|禁|差|不|多|30|年|，|因|此|他|无|法|到|所|属|堂|区|进|行|牧|灵|工|作|而|专|注|研|读|等|工|作|。|范|廷|颂|除|了|面|对|战|争|、|贫|困|、|被|当|局|迫|害|天|主|教|会|等|问|题|外|，|也|秘|密|恢|复|修|院|、|创|建|女|修|会|团|体|等|。|1990|年|，|教|宗|若|望|保|禄|二|世|在|同|年|6|月|18|日|擢|升|范|廷|颂|为|天|主|教|河|内|总|教|区|宗|座|署|[SEP]'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:23.246106Z",
     "start_time": "2025-05-20T13:52:23.242954Z"
    }
   },
   "cell_type": "code",
   "source": "print(list(zip(tokenized_examples[\"input_ids\"][0], tokenized_examples[\"token_type_ids\"][0])))",
   "id": "e291c3bdc98170d4",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(101, 0), (5745, 0), (2455, 0), (7563, 0), (3221, 0), (784, 0), (720, 0), (3198, 0), (952, 0), (6158, 0), (818, 0), (711, 0), (712, 0), (3136, 0), (4638, 0), (8043, 0), (102, 0), (5745, 1), (2455, 1), (7563, 1), (3364, 1), (3322, 1), (8020, 1), (8024, 1), (8021, 1), (8024, 1), (1760, 1), (1399, 1), (924, 1), (4882, 1), (185, 1), (5735, 1), (4449, 1), (8020, 1), (8021, 1), (8024, 1), (3221, 1), (6632, 1), (1298, 1), (5384, 1), (7716, 1), (1921, 1), (712, 1), (3136, 1), (3364, 1), (3322, 1), (511, 1), (9155, 1), (2399, 1), (6158, 1), (818, 1), (711, 1), (712, 1), (3136, 1), (8039, 1), (8431, 1), (2399, 1), (6158, 1), (3091, 1), (1285, 1), (711, 1), (1921, 1), (712, 1), (3136, 1), (3777, 1), (1079, 1), (2600, 1), (3136, 1), (1277, 1), (2134, 1), (2429, 1), (5392, 1), (4415, 1), (8039, 1), (8447, 1), (2399, 1), (6158, 1), (3091, 1), (1285, 1), (711, 1), (2600, 1), (712, 1), (3136, 1), (8024, 1), (1398, 1), (2399, 1), (2399, 1), (2419, 1), (6158, 1), (3091, 1), (1285, 1), (711, 1), (3364, 1), (3322, 1), (8039, 1), (8170, 1), (2399, 1), (123, 1), (3299, 1), (4895, 1), (686, 1), (511, 1), (5745, 1), (2455, 1), (7563, 1), (754, 1), (9915, 1), (2399, 1), (127, 1), (3299, 1), (8115, 1), (3189, 1), (1762, 1), (6632, 1), (1298, 1), (2123, 1), (2398, 1), (4689, 1), (1921, 1), (712, 1), (3136, 1), (1355, 1), (5683, 1), (3136, 1), (1277, 1), (1139, 1), (4495, 1), (8039, 1), (4997, 1), (2399, 1), (3198, 1), (2970, 1), (1358, 1), (5679, 1), (1962, 1), (3136, 1), (5509, 1), (1400, 1), (8024, 1), (6158, 1), (671, 1), (855, 1), (6632, 1), (1298, 1), (4868, 1), (4266, 1), (2372, 1), (1168, 1), (3777, 1), (1079, 1), (5326, 1), (5330, 1), (1071, 1), (2110, 1), (689, 1), (511, 1), (5745, 1), (2455, 1), (7563, 1), (754, 1), (9211, 1), (2399, 1), (1762, 1), (3777, 1), (1079, 1), (1920, 1), (934, 1), (6887, 1), (7368, 1), (2130, 1), (2768, 1), (4868, 1), (2110, 1), (2110, 1), (689, 1), (511, 1), (5745, 1), (2455, 1), (7563, 1), (754, 1), (8594, 1), (2399, 1), (127, 1), (3299, 1), (127, 1), (3189, 1), (1762, 1), (3777, 1), (1079, 1), (4638, 1), (712, 1), (3136, 1), (2429, 1), (1828, 1), (3232, 1), (7195, 1), (8039, 1), (1350, 1), (1400, 1), (6158, 1), (3836, 1), (1168, 1), (1760, 1), (1957, 1), (2207, 1), (2548, 1), (1065, 1), (2109, 1), (1036, 1), (7368, 1), (3302, 1), (1218, 1), (511, 1), (8707, 1), (2399, 1), (807, 1), (8024, 1), (5745, 1), (2455, 1), (7563, 1), (1762, 1), (3777, 1), (1079, 1), (1828, 1), (1277, 1), (1158, 1), (2456, 1), (4919, 1), (3696, 1), (2970, 1), (2521, 1), (704, 1), (2552, 1), (809, 1), (3119, 1), (2159, 1), (1168, 1), (3777, 1), (1079, 1), (6912, 1), (2773, 1), (4638, 1), (7410, 1), (3696, 1), (511, 1), (9258, 1), (2399, 1), (8024, 1), (3791, 1), (6632, 1), (2773, 1), (751, 1), (5310, 1), (3338, 1), (8024, 1), (6632, 1), (1298, 1), (3696, 1), (712, 1), (1066, 1), (1469, 1), (1744, 1), (2456, 1), (6963, 1), (3777, 1), (1079, 1), (8024, 1), (2496, 1), (3198, 1), (2523, 1), (1914, 1), (1921, 1), (712, 1), (3136, 1), (4868, 1), (5466, 1), (782, 1), (1447, 1), (6845, 1), (5635, 1), (6632, 1), (1298, 1), (4638, 1), (1298, 1), (3175, 1), (8024, 1), (852, 1), (5745, 1), (2455, 1), (7563, 1), (793, 1), (4197, 1), (4522, 1), (1762, 1), (3777, 1), (1079, 1), (511, 1), (5422, 1), (2399, 1), (5052, 1), (4415, 1), (1760, 1), (5735, 1), (3307, 1), (2207, 1), (934, 1), (7368, 1), (8039, 1), (2668, 1), (1762, 1), (8779, 1), (2399, 1), (1728, 1), (2932, 1), (1310, 1), (934, 1), (7368, 1), (4638, 1), (5632, 1), (4507, 1), (510, 1), (5632, 1), (3780, 1), (1350, 1), (2867, 1), (5318, 1), (3124, 1), (2424, 1), (1762, 1), (934, 1), (7368, 1), (6392, 1), (3124, 1), (3780, 1), (6440, 1), (4638, 1), (6206, 1), (3724, 1), (5445, 1), (6158, 1), (2936, 1), (511, 1), (9155, 1), (2399, 1), (125, 1), (3299, 1), (126, 1), (3189, 1), (8024, 1), (3136, 1), (2134, 1), (818, 1), (1462, 1), (5745, 1), (2455, 1), (7563, 1), (711, 1), (1921, 1), (712, 1), (3136, 1), (1266, 1), (2123, 1), (3136, 1), (1277, 1), (712, 1), (3136, 1), (8024, 1), (1398, 1), (2399, 1), (129, 1), (3299, 1), (8115, 1), (3189, 1), (2218, 1), (818, 1), (8039, 1), (1071, 1), (4288, 1), (7208, 1), (711, 1), (519, 1), (2769, 1), (928, 1), (1921, 1), (712, 1), (4638, 1), (4263, 1), (520, 1), (511, 1), (4507, 1), (754, 1), (5745, 1), (2455, 1), (7563, 1), (6158, 1), (6632, 1), (1298, 1), (3124, 1), (2424, 1), (6763, 1), (4881, 1), (2345, 1), (679, 1), (1914, 1), (8114, 1), (2399, 1), (8024, 1), (1728, 1), (3634, 1), (800, 1), (3187, 1), (3791, 1), (1168, 1), (2792, 1), (2247, 1), (1828, 1), (1277, 1), (6822, 1), (6121, 1), (4288, 1), (4130, 1), (2339, 1), (868, 1), (5445, 1), (683, 1), (3800, 1), (4777, 1), (6438, 1), (5023, 1), (2339, 1), (868, 1), (511, 1), (5745, 1), (2455, 1), (7563, 1), (7370, 1), (749, 1), (7481, 1), (2190, 1), (2773, 1), (751, 1), (510, 1), (6577, 1), (1737, 1), (510, 1), (6158, 1), (2496, 1), (2229, 1), (6833, 1), (2154, 1), (1921, 1), (712, 1), (3136, 1), (833, 1), (5023, 1), (7309, 1), (7579, 1), (1912, 1), (8024, 1), (738, 1), (4908, 1), (2166, 1), (2612, 1), (1908, 1), (934, 1), (7368, 1), (510, 1), (1158, 1), (2456, 1), (1957, 1), (934, 1), (833, 1), (1730, 1), (860, 1), (5023, 1), (511, 1), (8431, 1), (2399, 1), (8024, 1), (3136, 1), (2134, 1), (5735, 1), (3307, 1), (924, 1), (4882, 1), (753, 1), (686, 1), (1762, 1), (1398, 1), (2399, 1), (127, 1), (3299, 1), (8123, 1), (3189, 1), (3091, 1), (1285, 1), (5745, 1), (2455, 1), (7563, 1), (711, 1), (1921, 1), (712, 1), (3136, 1), (3777, 1), (1079, 1), (2600, 1), (3136, 1), (1277, 1), (2134, 1), (2429, 1), (5392, 1), (102, 1)]\n"
     ]
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:23.280214Z",
     "start_time": "2025-05-20T13:52:23.277112Z"
    }
   },
   "cell_type": "code",
   "source": [
    "print(tokenized_examples.sequence_ids(0), len(tokenized_examples.sequence_ids(0)))\n",
    "print(tokenized_examples[\"offset_mapping\"][0])"
   ],
   "id": "d7c7dc2681348126",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[None, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, None, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, None] 512\n",
      "[(0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (0, 0), (0, 1), (1, 2), (2, 3), (3, 4), (4, 5), (5, 6), (6, 7), (7, 8), (8, 9), (9, 10), (10, 11), (11, 12), (12, 13), (13, 14), (14, 15), (15, 16), (16, 17), (17, 18), (18, 19), (19, 20), (20, 21), (21, 22), (22, 23), (23, 24), (24, 25), (25, 26), (26, 27), (27, 28), (28, 29), (29, 30), (30, 34), (34, 35), (35, 36), (36, 37), (37, 38), (38, 39), (39, 40), (40, 41), (41, 45), (45, 46), (46, 47), (47, 48), (48, 49), (49, 50), (50, 51), (51, 52), (52, 53), (53, 54), (54, 55), (55, 56), (56, 57), (57, 58), (58, 59), (59, 60), (60, 61), (61, 62), (62, 63), (63, 67), (67, 68), (68, 69), (69, 70), (70, 71), (71, 72), (72, 73), (73, 74), (74, 75), (75, 76), (76, 77), (77, 78), (78, 79), (79, 80), (80, 81), (81, 82), (82, 83), (83, 84), (84, 85), (85, 86), (86, 87), (87, 91), (91, 92), (92, 93), (93, 94), (94, 95), (95, 96), (96, 97), (97, 98), (98, 99), (99, 100), (100, 101), (101, 105), (105, 106), (106, 107), (107, 108), (108, 110), (110, 111), (111, 112), (112, 113), (113, 114), (114, 115), (115, 116), (116, 117), (117, 118), (118, 119), (119, 120), (120, 121), (121, 122), (122, 123), (123, 124), (124, 125), (125, 126), (126, 127), (127, 128), (128, 129), (129, 130), (130, 131), (131, 132), (132, 133), (133, 134), (134, 135), (135, 136), (136, 137), (137, 138), (138, 139), (139, 140), (140, 141), (141, 142), (142, 143), (143, 144), (144, 145), (145, 146), (146, 147), (147, 148), (148, 149), (149, 150), (150, 151), (151, 152), (152, 153), (153, 154), (154, 155), (155, 156), (156, 157), (157, 158), (158, 159), (159, 163), (163, 164), (164, 165), (165, 166), (166, 167), (167, 168), (168, 169), (169, 170), (170, 171), (171, 172), (172, 173), (173, 174), (174, 175), (175, 176), (176, 177), (177, 178), (178, 179), (179, 180), (180, 181), (181, 182), (182, 186), (186, 187), (187, 188), (188, 189), (189, 190), (190, 191), (191, 192), (192, 193), (193, 194), (194, 195), (195, 196), (196, 197), (197, 198), (198, 199), (199, 200), (200, 201), (201, 202), (202, 203), (203, 204), (204, 205), (205, 206), (206, 207), (207, 208), (208, 209), (209, 210), (210, 211), (211, 212), (212, 213), (213, 214), (214, 215), (215, 216), (216, 217), (217, 218), (218, 222), (222, 223), (223, 224), (224, 225), (225, 226), (226, 227), (227, 228), (228, 229), (229, 230), (230, 231), (231, 232), (232, 233), (233, 234), (234, 235), (235, 236), (236, 237), (237, 238), (238, 239), (239, 240), (240, 241), (241, 242), (242, 243), (243, 244), (244, 245), (245, 246), (246, 247), (247, 248), (248, 249), (249, 250), (250, 251), (251, 252), (252, 253), (253, 257), (257, 258), (258, 259), (259, 260), (260, 261), (261, 262), (262, 263), (263, 264), (264, 265), (265, 266), (266, 267), (267, 268), (268, 269), (269, 270), (270, 271), (271, 272), (272, 273), (273, 274), (274, 275), (275, 276), (276, 277), (277, 278), (278, 279), (279, 280), (280, 281), (281, 282), (282, 283), (283, 284), (284, 285), (285, 286), (286, 287), (287, 288), (288, 289), (289, 290), (290, 291), (291, 292), (292, 293), (293, 294), (294, 295), (295, 296), (296, 297), (297, 298), (298, 299), (299, 300), (300, 301), (301, 302), (302, 303), (303, 304), (304, 305), (305, 306), (306, 307), (307, 308), (308, 309), (309, 310), (310, 311), (311, 312), (312, 313), (313, 314), (314, 315), (315, 316), (316, 317), (317, 318), (318, 319), (319, 320), (320, 321), (321, 325), (325, 326), (326, 327), (327, 328), (328, 329), (329, 330), (330, 331), (331, 332), (332, 333), (333, 334), (334, 335), (335, 336), (336, 337), (337, 338), (338, 339), (339, 340), (340, 341), (341, 342), (342, 343), (343, 344), (344, 345), (345, 346), (346, 347), (347, 348), (348, 349), (349, 350), (350, 351), (351, 352), (352, 353), (353, 354), (354, 355), (355, 356), (356, 360), (360, 361), (361, 362), (362, 363), (363, 364), (364, 365), (365, 366), (366, 367), (367, 368), (368, 369), (369, 370), (370, 371), (371, 372), (372, 373), (373, 374), (374, 375), (375, 376), (376, 377), (377, 378), (378, 379), (379, 380), (380, 381), (381, 382), (382, 383), (383, 384), (384, 385), (385, 386), (386, 387), (387, 388), (388, 390), (390, 391), (391, 392), (392, 393), (393, 394), (394, 395), (395, 396), (396, 397), (397, 398), (398, 399), (399, 400), (400, 401), (401, 402), (402, 403), (403, 404), (404, 405), (405, 406), (406, 407), (407, 408), (408, 409), (409, 410), (410, 411), (411, 412), (412, 413), (413, 414), (414, 415), (415, 416), (416, 417), (417, 418), (418, 419), (419, 420), (420, 421), (421, 422), (422, 424), (424, 425), (425, 426), (426, 427), (427, 428), (428, 429), (429, 430), (430, 431), (431, 432), (432, 433), (433, 434), (434, 435), (435, 436), (436, 437), (437, 438), (438, 439), (439, 440), (440, 441), (441, 442), (442, 443), (443, 444), (444, 445), (445, 446), (446, 447), (447, 448), (448, 449), (449, 450), (450, 451), (451, 452), (452, 453), (453, 454), (454, 455), (455, 456), (456, 457), (457, 458), (458, 459), (459, 460), (460, 461), (461, 462), (462, 463), (463, 464), (464, 465), (465, 466), (466, 467), (467, 468), (468, 469), (469, 470), (470, 471), (471, 472), (472, 473), (473, 474), (474, 475), (475, 476), (476, 477), (477, 478), (478, 479), (479, 480), (480, 481), (481, 482), (482, 483), (483, 484), (484, 485), (485, 486), (486, 487), (487, 488), (488, 489), (489, 490), (490, 491), (491, 492), (492, 493), (493, 494), (494, 495), (495, 499), (499, 500), (500, 501), (501, 502), (502, 503), (503, 504), (504, 505), (505, 506), (506, 507), (507, 508), (508, 509), (509, 510), (510, 511), (511, 512), (512, 513), (513, 514), (514, 516), (516, 517), (517, 518), (518, 519), (519, 520), (520, 521), (521, 522), (522, 523), (523, 524), (524, 525), (525, 526), (526, 527), (527, 528), (528, 529), (529, 530), (530, 531), (531, 532), (532, 533), (533, 534), (0, 0)]\n"
     ]
    }
   ],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:23.323515Z",
     "start_time": "2025-05-20T13:52:23.310055Z"
    }
   },
   "cell_type": "code",
   "source": "list(zip(tokenized_examples.sequence_ids(0), tokenized_examples[\"offset_mapping\"][0]))",
   "id": "4836d13a4dc4e78b",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(None, (0, 0)),\n",
       " (0, (0, 1)),\n",
       " (0, (1, 2)),\n",
       " (0, (2, 3)),\n",
       " (0, (3, 4)),\n",
       " (0, (4, 5)),\n",
       " (0, (5, 6)),\n",
       " (0, (6, 7)),\n",
       " (0, (7, 8)),\n",
       " (0, (8, 9)),\n",
       " (0, (9, 10)),\n",
       " (0, (10, 11)),\n",
       " (0, (11, 12)),\n",
       " (0, (12, 13)),\n",
       " (0, (13, 14)),\n",
       " (0, (14, 15)),\n",
       " (None, (0, 0)),\n",
       " (1, (0, 1)),\n",
       " (1, (1, 2)),\n",
       " (1, (2, 3)),\n",
       " (1, (3, 4)),\n",
       " (1, (4, 5)),\n",
       " (1, (5, 6)),\n",
       " (1, (6, 7)),\n",
       " (1, (7, 8)),\n",
       " (1, (8, 9)),\n",
       " (1, (9, 10)),\n",
       " (1, (10, 11)),\n",
       " (1, (11, 12)),\n",
       " (1, (12, 13)),\n",
       " (1, (13, 14)),\n",
       " (1, (14, 15)),\n",
       " (1, (15, 16)),\n",
       " (1, (16, 17)),\n",
       " (1, (17, 18)),\n",
       " (1, (18, 19)),\n",
       " (1, (19, 20)),\n",
       " (1, (20, 21)),\n",
       " (1, (21, 22)),\n",
       " (1, (22, 23)),\n",
       " (1, (23, 24)),\n",
       " (1, (24, 25)),\n",
       " (1, (25, 26)),\n",
       " (1, (26, 27)),\n",
       " (1, (27, 28)),\n",
       " (1, (28, 29)),\n",
       " (1, (29, 30)),\n",
       " (1, (30, 34)),\n",
       " (1, (34, 35)),\n",
       " (1, (35, 36)),\n",
       " (1, (36, 37)),\n",
       " (1, (37, 38)),\n",
       " (1, (38, 39)),\n",
       " (1, (39, 40)),\n",
       " (1, (40, 41)),\n",
       " (1, (41, 45)),\n",
       " (1, (45, 46)),\n",
       " (1, (46, 47)),\n",
       " (1, (47, 48)),\n",
       " (1, (48, 49)),\n",
       " (1, (49, 50)),\n",
       " (1, (50, 51)),\n",
       " (1, (51, 52)),\n",
       " (1, (52, 53)),\n",
       " (1, (53, 54)),\n",
       " (1, (54, 55)),\n",
       " (1, (55, 56)),\n",
       " (1, (56, 57)),\n",
       " (1, (57, 58)),\n",
       " (1, (58, 59)),\n",
       " (1, (59, 60)),\n",
       " (1, (60, 61)),\n",
       " (1, (61, 62)),\n",
       " (1, (62, 63)),\n",
       " (1, (63, 67)),\n",
       " (1, (67, 68)),\n",
       " (1, (68, 69)),\n",
       " (1, (69, 70)),\n",
       " (1, (70, 71)),\n",
       " (1, (71, 72)),\n",
       " (1, (72, 73)),\n",
       " (1, (73, 74)),\n",
       " (1, (74, 75)),\n",
       " (1, (75, 76)),\n",
       " (1, (76, 77)),\n",
       " (1, (77, 78)),\n",
       " (1, (78, 79)),\n",
       " (1, (79, 80)),\n",
       " (1, (80, 81)),\n",
       " (1, (81, 82)),\n",
       " (1, (82, 83)),\n",
       " (1, (83, 84)),\n",
       " (1, (84, 85)),\n",
       " (1, (85, 86)),\n",
       " (1, (86, 87)),\n",
       " (1, (87, 91)),\n",
       " (1, (91, 92)),\n",
       " (1, (92, 93)),\n",
       " (1, (93, 94)),\n",
       " (1, (94, 95)),\n",
       " (1, (95, 96)),\n",
       " (1, (96, 97)),\n",
       " (1, (97, 98)),\n",
       " (1, (98, 99)),\n",
       " (1, (99, 100)),\n",
       " (1, (100, 101)),\n",
       " (1, (101, 105)),\n",
       " (1, (105, 106)),\n",
       " (1, (106, 107)),\n",
       " (1, (107, 108)),\n",
       " (1, (108, 110)),\n",
       " (1, (110, 111)),\n",
       " (1, (111, 112)),\n",
       " (1, (112, 113)),\n",
       " (1, (113, 114)),\n",
       " (1, (114, 115)),\n",
       " (1, (115, 116)),\n",
       " (1, (116, 117)),\n",
       " (1, (117, 118)),\n",
       " (1, (118, 119)),\n",
       " (1, (119, 120)),\n",
       " (1, (120, 121)),\n",
       " (1, (121, 122)),\n",
       " (1, (122, 123)),\n",
       " (1, (123, 124)),\n",
       " (1, (124, 125)),\n",
       " (1, (125, 126)),\n",
       " (1, (126, 127)),\n",
       " (1, (127, 128)),\n",
       " (1, (128, 129)),\n",
       " (1, (129, 130)),\n",
       " (1, (130, 131)),\n",
       " (1, (131, 132)),\n",
       " (1, (132, 133)),\n",
       " (1, (133, 134)),\n",
       " (1, (134, 135)),\n",
       " (1, (135, 136)),\n",
       " (1, (136, 137)),\n",
       " (1, (137, 138)),\n",
       " (1, (138, 139)),\n",
       " (1, (139, 140)),\n",
       " (1, (140, 141)),\n",
       " (1, (141, 142)),\n",
       " (1, (142, 143)),\n",
       " (1, (143, 144)),\n",
       " (1, (144, 145)),\n",
       " (1, (145, 146)),\n",
       " (1, (146, 147)),\n",
       " (1, (147, 148)),\n",
       " (1, (148, 149)),\n",
       " (1, (149, 150)),\n",
       " (1, (150, 151)),\n",
       " (1, (151, 152)),\n",
       " (1, (152, 153)),\n",
       " (1, (153, 154)),\n",
       " (1, (154, 155)),\n",
       " (1, (155, 156)),\n",
       " (1, (156, 157)),\n",
       " (1, (157, 158)),\n",
       " (1, (158, 159)),\n",
       " (1, (159, 163)),\n",
       " (1, (163, 164)),\n",
       " (1, (164, 165)),\n",
       " (1, (165, 166)),\n",
       " (1, (166, 167)),\n",
       " (1, (167, 168)),\n",
       " (1, (168, 169)),\n",
       " (1, (169, 170)),\n",
       " (1, (170, 171)),\n",
       " (1, (171, 172)),\n",
       " (1, (172, 173)),\n",
       " (1, (173, 174)),\n",
       " (1, (174, 175)),\n",
       " (1, (175, 176)),\n",
       " (1, (176, 177)),\n",
       " (1, (177, 178)),\n",
       " (1, (178, 179)),\n",
       " (1, (179, 180)),\n",
       " (1, (180, 181)),\n",
       " (1, (181, 182)),\n",
       " (1, (182, 186)),\n",
       " (1, (186, 187)),\n",
       " (1, (187, 188)),\n",
       " (1, (188, 189)),\n",
       " (1, (189, 190)),\n",
       " (1, (190, 191)),\n",
       " (1, (191, 192)),\n",
       " (1, (192, 193)),\n",
       " (1, (193, 194)),\n",
       " (1, (194, 195)),\n",
       " (1, (195, 196)),\n",
       " (1, (196, 197)),\n",
       " (1, (197, 198)),\n",
       " (1, (198, 199)),\n",
       " (1, (199, 200)),\n",
       " (1, (200, 201)),\n",
       " (1, (201, 202)),\n",
       " (1, (202, 203)),\n",
       " (1, (203, 204)),\n",
       " (1, (204, 205)),\n",
       " (1, (205, 206)),\n",
       " (1, (206, 207)),\n",
       " (1, (207, 208)),\n",
       " (1, (208, 209)),\n",
       " (1, (209, 210)),\n",
       " (1, (210, 211)),\n",
       " (1, (211, 212)),\n",
       " (1, (212, 213)),\n",
       " (1, (213, 214)),\n",
       " (1, (214, 215)),\n",
       " (1, (215, 216)),\n",
       " (1, (216, 217)),\n",
       " (1, (217, 218)),\n",
       " (1, (218, 222)),\n",
       " (1, (222, 223)),\n",
       " (1, (223, 224)),\n",
       " (1, (224, 225)),\n",
       " (1, (225, 226)),\n",
       " (1, (226, 227)),\n",
       " (1, (227, 228)),\n",
       " (1, (228, 229)),\n",
       " (1, (229, 230)),\n",
       " (1, (230, 231)),\n",
       " (1, (231, 232)),\n",
       " (1, (232, 233)),\n",
       " (1, (233, 234)),\n",
       " (1, (234, 235)),\n",
       " (1, (235, 236)),\n",
       " (1, (236, 237)),\n",
       " (1, (237, 238)),\n",
       " (1, (238, 239)),\n",
       " (1, (239, 240)),\n",
       " (1, (240, 241)),\n",
       " (1, (241, 242)),\n",
       " (1, (242, 243)),\n",
       " (1, (243, 244)),\n",
       " (1, (244, 245)),\n",
       " (1, (245, 246)),\n",
       " (1, (246, 247)),\n",
       " (1, (247, 248)),\n",
       " (1, (248, 249)),\n",
       " (1, (249, 250)),\n",
       " (1, (250, 251)),\n",
       " (1, (251, 252)),\n",
       " (1, (252, 253)),\n",
       " (1, (253, 257)),\n",
       " (1, (257, 258)),\n",
       " (1, (258, 259)),\n",
       " (1, (259, 260)),\n",
       " (1, (260, 261)),\n",
       " (1, (261, 262)),\n",
       " (1, (262, 263)),\n",
       " (1, (263, 264)),\n",
       " (1, (264, 265)),\n",
       " (1, (265, 266)),\n",
       " (1, (266, 267)),\n",
       " (1, (267, 268)),\n",
       " (1, (268, 269)),\n",
       " (1, (269, 270)),\n",
       " (1, (270, 271)),\n",
       " (1, (271, 272)),\n",
       " (1, (272, 273)),\n",
       " (1, (273, 274)),\n",
       " (1, (274, 275)),\n",
       " (1, (275, 276)),\n",
       " (1, (276, 277)),\n",
       " (1, (277, 278)),\n",
       " (1, (278, 279)),\n",
       " (1, (279, 280)),\n",
       " (1, (280, 281)),\n",
       " (1, (281, 282)),\n",
       " (1, (282, 283)),\n",
       " (1, (283, 284)),\n",
       " (1, (284, 285)),\n",
       " (1, (285, 286)),\n",
       " (1, (286, 287)),\n",
       " (1, (287, 288)),\n",
       " (1, (288, 289)),\n",
       " (1, (289, 290)),\n",
       " (1, (290, 291)),\n",
       " (1, (291, 292)),\n",
       " (1, (292, 293)),\n",
       " (1, (293, 294)),\n",
       " (1, (294, 295)),\n",
       " (1, (295, 296)),\n",
       " (1, (296, 297)),\n",
       " (1, (297, 298)),\n",
       " (1, (298, 299)),\n",
       " (1, (299, 300)),\n",
       " (1, (300, 301)),\n",
       " (1, (301, 302)),\n",
       " (1, (302, 303)),\n",
       " (1, (303, 304)),\n",
       " (1, (304, 305)),\n",
       " (1, (305, 306)),\n",
       " (1, (306, 307)),\n",
       " (1, (307, 308)),\n",
       " (1, (308, 309)),\n",
       " (1, (309, 310)),\n",
       " (1, (310, 311)),\n",
       " (1, (311, 312)),\n",
       " (1, (312, 313)),\n",
       " (1, (313, 314)),\n",
       " (1, (314, 315)),\n",
       " (1, (315, 316)),\n",
       " (1, (316, 317)),\n",
       " (1, (317, 318)),\n",
       " (1, (318, 319)),\n",
       " (1, (319, 320)),\n",
       " (1, (320, 321)),\n",
       " (1, (321, 325)),\n",
       " (1, (325, 326)),\n",
       " (1, (326, 327)),\n",
       " (1, (327, 328)),\n",
       " (1, (328, 329)),\n",
       " (1, (329, 330)),\n",
       " (1, (330, 331)),\n",
       " (1, (331, 332)),\n",
       " (1, (332, 333)),\n",
       " (1, (333, 334)),\n",
       " (1, (334, 335)),\n",
       " (1, (335, 336)),\n",
       " (1, (336, 337)),\n",
       " (1, (337, 338)),\n",
       " (1, (338, 339)),\n",
       " (1, (339, 340)),\n",
       " (1, (340, 341)),\n",
       " (1, (341, 342)),\n",
       " (1, (342, 343)),\n",
       " (1, (343, 344)),\n",
       " (1, (344, 345)),\n",
       " (1, (345, 346)),\n",
       " (1, (346, 347)),\n",
       " (1, (347, 348)),\n",
       " (1, (348, 349)),\n",
       " (1, (349, 350)),\n",
       " (1, (350, 351)),\n",
       " (1, (351, 352)),\n",
       " (1, (352, 353)),\n",
       " (1, (353, 354)),\n",
       " (1, (354, 355)),\n",
       " (1, (355, 356)),\n",
       " (1, (356, 360)),\n",
       " (1, (360, 361)),\n",
       " (1, (361, 362)),\n",
       " (1, (362, 363)),\n",
       " (1, (363, 364)),\n",
       " (1, (364, 365)),\n",
       " (1, (365, 366)),\n",
       " (1, (366, 367)),\n",
       " (1, (367, 368)),\n",
       " (1, (368, 369)),\n",
       " (1, (369, 370)),\n",
       " (1, (370, 371)),\n",
       " (1, (371, 372)),\n",
       " (1, (372, 373)),\n",
       " (1, (373, 374)),\n",
       " (1, (374, 375)),\n",
       " (1, (375, 376)),\n",
       " (1, (376, 377)),\n",
       " (1, (377, 378)),\n",
       " (1, (378, 379)),\n",
       " (1, (379, 380)),\n",
       " (1, (380, 381)),\n",
       " (1, (381, 382)),\n",
       " (1, (382, 383)),\n",
       " (1, (383, 384)),\n",
       " (1, (384, 385)),\n",
       " (1, (385, 386)),\n",
       " (1, (386, 387)),\n",
       " (1, (387, 388)),\n",
       " (1, (388, 390)),\n",
       " (1, (390, 391)),\n",
       " (1, (391, 392)),\n",
       " (1, (392, 393)),\n",
       " (1, (393, 394)),\n",
       " (1, (394, 395)),\n",
       " (1, (395, 396)),\n",
       " (1, (396, 397)),\n",
       " (1, (397, 398)),\n",
       " (1, (398, 399)),\n",
       " (1, (399, 400)),\n",
       " (1, (400, 401)),\n",
       " (1, (401, 402)),\n",
       " (1, (402, 403)),\n",
       " (1, (403, 404)),\n",
       " (1, (404, 405)),\n",
       " (1, (405, 406)),\n",
       " (1, (406, 407)),\n",
       " (1, (407, 408)),\n",
       " (1, (408, 409)),\n",
       " (1, (409, 410)),\n",
       " (1, (410, 411)),\n",
       " (1, (411, 412)),\n",
       " (1, (412, 413)),\n",
       " (1, (413, 414)),\n",
       " (1, (414, 415)),\n",
       " (1, (415, 416)),\n",
       " (1, (416, 417)),\n",
       " (1, (417, 418)),\n",
       " (1, (418, 419)),\n",
       " (1, (419, 420)),\n",
       " (1, (420, 421)),\n",
       " (1, (421, 422)),\n",
       " (1, (422, 424)),\n",
       " (1, (424, 425)),\n",
       " (1, (425, 426)),\n",
       " (1, (426, 427)),\n",
       " (1, (427, 428)),\n",
       " (1, (428, 429)),\n",
       " (1, (429, 430)),\n",
       " (1, (430, 431)),\n",
       " (1, (431, 432)),\n",
       " (1, (432, 433)),\n",
       " (1, (433, 434)),\n",
       " (1, (434, 435)),\n",
       " (1, (435, 436)),\n",
       " (1, (436, 437)),\n",
       " (1, (437, 438)),\n",
       " (1, (438, 439)),\n",
       " (1, (439, 440)),\n",
       " (1, (440, 441)),\n",
       " (1, (441, 442)),\n",
       " (1, (442, 443)),\n",
       " (1, (443, 444)),\n",
       " (1, (444, 445)),\n",
       " (1, (445, 446)),\n",
       " (1, (446, 447)),\n",
       " (1, (447, 448)),\n",
       " (1, (448, 449)),\n",
       " (1, (449, 450)),\n",
       " (1, (450, 451)),\n",
       " (1, (451, 452)),\n",
       " (1, (452, 453)),\n",
       " (1, (453, 454)),\n",
       " (1, (454, 455)),\n",
       " (1, (455, 456)),\n",
       " (1, (456, 457)),\n",
       " (1, (457, 458)),\n",
       " (1, (458, 459)),\n",
       " (1, (459, 460)),\n",
       " (1, (460, 461)),\n",
       " (1, (461, 462)),\n",
       " (1, (462, 463)),\n",
       " (1, (463, 464)),\n",
       " (1, (464, 465)),\n",
       " (1, (465, 466)),\n",
       " (1, (466, 467)),\n",
       " (1, (467, 468)),\n",
       " (1, (468, 469)),\n",
       " (1, (469, 470)),\n",
       " (1, (470, 471)),\n",
       " (1, (471, 472)),\n",
       " (1, (472, 473)),\n",
       " (1, (473, 474)),\n",
       " (1, (474, 475)),\n",
       " (1, (475, 476)),\n",
       " (1, (476, 477)),\n",
       " (1, (477, 478)),\n",
       " (1, (478, 479)),\n",
       " (1, (479, 480)),\n",
       " (1, (480, 481)),\n",
       " (1, (481, 482)),\n",
       " (1, (482, 483)),\n",
       " (1, (483, 484)),\n",
       " (1, (484, 485)),\n",
       " (1, (485, 486)),\n",
       " (1, (486, 487)),\n",
       " (1, (487, 488)),\n",
       " (1, (488, 489)),\n",
       " (1, (489, 490)),\n",
       " (1, (490, 491)),\n",
       " (1, (491, 492)),\n",
       " (1, (492, 493)),\n",
       " (1, (493, 494)),\n",
       " (1, (494, 495)),\n",
       " (1, (495, 499)),\n",
       " (1, (499, 500)),\n",
       " (1, (500, 501)),\n",
       " (1, (501, 502)),\n",
       " (1, (502, 503)),\n",
       " (1, (503, 504)),\n",
       " (1, (504, 505)),\n",
       " (1, (505, 506)),\n",
       " (1, (506, 507)),\n",
       " (1, (507, 508)),\n",
       " (1, (508, 509)),\n",
       " (1, (509, 510)),\n",
       " (1, (510, 511)),\n",
       " (1, (511, 512)),\n",
       " (1, (512, 513)),\n",
       " (1, (513, 514)),\n",
       " (1, (514, 516)),\n",
       " (1, (516, 517)),\n",
       " (1, (517, 518)),\n",
       " (1, (518, 519)),\n",
       " (1, (519, 520)),\n",
       " (1, (520, 521)),\n",
       " (1, (521, 522)),\n",
       " (1, (522, 523)),\n",
       " (1, (523, 524)),\n",
       " (1, (524, 525)),\n",
       " (1, (525, 526)),\n",
       " (1, (526, 527)),\n",
       " (1, (527, 528)),\n",
       " (1, (528, 529)),\n",
       " (1, (529, 530)),\n",
       " (1, (530, 531)),\n",
       " (1, (531, 532)),\n",
       " (1, (532, 533)),\n",
       " (1, (533, 534)),\n",
       " (None, (0, 0))]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:23.364435Z",
     "start_time": "2025-05-20T13:52:23.359586Z"
    }
   },
   "cell_type": "code",
   "source": [
    "for idx, offset in enumerate(tokenized_examples[\"offset_mapping\"]):\n",
    "    data = sample_dataset[idx]\n",
    "    answers = data[\"answers\"]\n",
    "    # 答案所在位置\n",
    "    answer_start = answers[\"answer_start\"][0]\n",
    "    answer_end = answer_start + len(answers[\"text\"][0])\n",
    "\n",
    "    seq_ids = tokenized_examples.sequence_ids(idx)\n",
    "    # 查找第一个1位置\n",
    "    token_start = seq_ids.index(1)\n",
    "    # 查找最后一个1的位置\n",
    "    token_end = seq_ids.index(None, token_start) - 1\n",
    "    # print(f'answers:{answers} :: answer_start:{answer_start} answer_end:{answer_end} token_start:{token_start} token_end:{token_end}')\n",
    "\n",
    "    # 截断在答案位置之前\n",
    "    # 截断在答案位置之后（滑动窗口场景）\n",
    "    if offset[token_end][1] < answer_start or offset[token_start][0] > answer_end:\n",
    "        start_token_pos = end_token_pos = 0\n",
    "    else:\n",
    "        start_token_pos = token_start\n",
    "        while start_token_pos <= token_end and offset[start_token_pos][0] < answer_start:\n",
    "            start_token_pos += 1\n",
    "\n",
    "        end_token_pos = token_end\n",
    "        while end_token_pos >= token_start and offset[end_token_pos][1] > answer_end:\n",
    "            end_token_pos -= 1\n",
    "\n",
    "    print(answers, answer_start, answer_end, token_start, token_end, start_token_pos, end_token_pos)\n",
    "    print(\"token answer decode:\", tokenizer.decode(tokenized_examples[\"input_ids\"][idx][start_token_pos: end_token_pos + 1]))\n",
    "\n"
   ],
   "id": "c102e6845931db23",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'text': ['1963年'], 'answer_start': [30]} 30 35 17 510 47 48\n",
      "token answer decode: 1963 年\n",
      "{'text': ['1990年被擢升为天主教河内总教区宗座署理'], 'answer_start': [41]} 41 62 15 510 53 70\n",
      "token answer decode: 1990 年 被 擢 升 为 天 主 教 河 内 总 教 区 宗 座 署 理\n"
     ]
    }
   ],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:23.383459Z",
     "start_time": "2025-05-20T13:52:23.379439Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def process_func(examples, tokenizer=tokenizer):\n",
    "    tokenized_examples = tokenizer(text=examples[\"question\"], text_pair=examples[\"context\"], max_length=384, return_offsets_mapping=True, truncation=\"only_second\", padding=\"max_length\")\n",
    "\n",
    "    offset_mapping = tokenized_examples[\"offset_mapping\"]\n",
    "    start_positions = []\n",
    "    end_positions = []\n",
    "\n",
    "    for idx, offset in enumerate(offset_mapping):\n",
    "        ansers = examples[\"answers\"][idx]\n",
    "        answer_start = ansers[\"answer_start\"][0]\n",
    "        answer_end = answer_start + len(ansers[\"text\"][0])\n",
    "\n",
    "        seq_ids = tokenized_examples.sequence_ids(idx)\n",
    "        token_start = seq_ids.index(1)\n",
    "        token_end = seq_ids.index(None, token_start) - 1\n",
    "\n",
    "        if offset[token_end][1] < answer_start or offset[token_start][0] > answer_end:\n",
    "            start_token_pos = end_token_pos = 0\n",
    "        else:\n",
    "            start_token_pos = token_start\n",
    "            while start_token_pos <= token_end and offset[start_token_pos][0] < answer_start:\n",
    "                start_token_pos += 1\n",
    "\n",
    "            end_token_pos = token_end\n",
    "            while end_token_pos >= token_start and offset[end_token_pos][1] > answer_end:\n",
    "                end_token_pos -= 1\n",
    "\n",
    "        start_positions.append(start_token_pos)\n",
    "        end_positions.append(end_token_pos)\n",
    "\n",
    "    tokenized_examples[\"start_positions\"] = start_positions\n",
    "    tokenized_examples[\"end_positions\"] = end_positions\n",
    "    return tokenized_examples"
   ],
   "id": "a776b8b3b81e9fb2",
   "outputs": [],
   "execution_count": 12
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:23.487904Z",
     "start_time": "2025-05-20T13:52:23.427943Z"
    }
   },
   "cell_type": "code",
   "source": [
    "tokenied_datasets = datasets.map(process_func, batched=True, remove_columns=datasets[\"train\"].column_names)\n",
    "tokenied_datasets"
   ],
   "id": "ea05f1adde3ddd0a",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'offset_mapping', 'start_positions', 'end_positions'],\n",
       "        num_rows: 10142\n",
       "    })\n",
       "    validation: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'offset_mapping', 'start_positions', 'end_positions'],\n",
       "        num_rows: 3219\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['input_ids', 'token_type_ids', 'attention_mask', 'offset_mapping', 'start_positions', 'end_positions'],\n",
       "        num_rows: 1002\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 13
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:24.440040Z",
     "start_time": "2025-05-20T13:52:23.491909Z"
    }
   },
   "cell_type": "code",
   "source": [
    "model = AutoModelForQuestionAnswering.from_pretrained(\"hfl/chinese-macbert-base\")\n",
    "model\n"
   ],
   "id": "a171b23bbb7c17dc",
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of BertForQuestionAnswering were not initialized from the model checkpoint at hfl/chinese-macbert-base and are newly initialized: ['qa_outputs.bias', 'qa_outputs.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "BertForQuestionAnswering(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(21128, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1, inplace=False)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0-11): 12 x BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSdpaSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1, inplace=False)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "            (intermediate_act_fn): GELUActivation()\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1, inplace=False)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (qa_outputs): Linear(in_features=768, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 14
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:24.459512Z",
     "start_time": "2025-05-20T13:52:24.457366Z"
    }
   },
   "cell_type": "code",
   "source": [
    "type(model)\n",
    "from transformers.models.bert.modeling_bert import BertForQuestionAnswering"
   ],
   "id": "f50f1ddc81e5fd7b",
   "outputs": [],
   "execution_count": 15
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2025-05-20T13:52:24.713630Z",
     "start_time": "2025-05-20T13:52:24.484023Z"
    }
   },
   "cell_type": "code",
   "source": [
    "args = TrainingArguments(\n",
    "    output_dir=\"output\",\n",
    "    per_device_train_batch_size=32,\n",
    "    per_device_eval_batch_size=64,\n",
    "    eval_strategy=\"steps\",\n",
    "    logging_steps=10,\n",
    "    max_steps=50,\n",
    "    eval_steps=20,\n",
    ")\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=args,\n",
    "    tokenizer=tokenizer,\n",
    "    train_dataset=tokenied_datasets[\"train\"],\n",
    "    eval_dataset=tokenied_datasets[\"validation\"],\n",
    "    data_collator=DefaultDataCollator()\n",
    ")\n",
    "\n",
    "# trainer.train()"
   ],
   "id": "fc6aad00fbc89cae",
   "outputs": [],
   "execution_count": 16
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
